package com.kang.webmagic.test;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.pipeline.FilePipeline;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.scheduler.BloomFilterDuplicateRemover;
import us.codecraft.webmagic.scheduler.QueueScheduler;
import us.codecraft.webmagic.scheduler.Scheduler;

public class JobProcessor implements PageProcessor {

    //解析页面
    public void process(Page page) {
        //解析返回的数据page，并且把解析的结果放到ResultItem中
        //css选择器
        page.putField("div", page.getHtml().css("div.mt h2").all());
        //Xpath
        page.putField("div2", page.getHtml().xpath("//div[@id=news_div]/ul/li/div/a"));

        //正则表达式
        page.putField("div3", page.getHtml().css("div#news_div a").regex(".*江苏.*").all());

        //处理结果的API
        page.putField("div3", page.getHtml().css("div#news_div a").regex(".*江苏.*").get());
        page.putField("div3", page.getHtml().css("div#news_div a").regex(".*江苏.*").toString());

        //获取链接
        page.addTargetRequests(page.getHtml().css("div#news_div").links().regex(".*9$").all());
        page.putField("url", page.getHtml().css("div.mt h1").all());

        page.addTargetRequest("https://www.jd.com/news.html?id=37319");
        page.addTargetRequest("https://www.jd.com/news.html?id=37319");
        page.addTargetRequest("https://www.jd.com/news.html?id=37319");
    }

    private Site site = Site.me()
            .setCharset("UTF-8")//编码
            .setSleepTime(1)//抓取间隔时间
            .setTimeOut(1000 * 10)//超时时间,单位毫秒
            .setRetrySleepTime(3000)//重试时间
            .setRetryTimes(3);//重试次数


    public Site getSite() {
        return site;
    }

    //主函数执行爬虫
    public static void main(String[] args) {
        Spider spider = Spider.create(new JobProcessor())
                //初始访问url地址
                .addUrl("https://www.jd.com/moreSubject.aspx")  //设置爬取数据的页面
                //.addPipeline(new FilePipeline("D:/webmagic/"))
                .thread(5)  //设置5个线程处理
                .setScheduler(new QueueScheduler().setDuplicateRemover(new BloomFilterDuplicateRemover(10000000)));//设置布隆去重过滤器，指定最多对1000万数据进行去重操作

        Scheduler scheduler = spider.getScheduler();
        //执行爬虫
        spider.run();
    }

}
